# coding: utf-8
import urllib2
import string, model
import re, sys, datetime
from model import TradableStockHolder, Stock, HolderQuarterReport

class StockTraderGrabber:
    # top ten tradable shareholders
    # date is only 2010-06-30, 2011-03-15, 2010-12-31, 2010-12-31
    tradable_top_ten_link = string.Template(
        'http://quotef10.eastmoney.com/F10_ltgdtop10/$stockid.html'
    )
    
    # later maybe we should use the &time to time=195950
    # type 0 means sort by stock_id
    stock_list_link = string.Template(
        'http://quote.tool.hexun.com/hqzx/quote.aspx?type=2&market=0&sorttype=0&updown=down&page=$page&count=$count'
        )
    
    holder_amount_link = string.Template('http://stockdata.stock.hexun.com/f10/gudongrenshu.aspx?stockid=$stockid')
    
    #u'2011年04月22日'
    quarter_date_re = re.compile(u'(\d{4})年(\d{2})月(\d{2})日')
    
    def grabStockList(self):
        cur_page = 1
        stock_item_re = re.compile(u'\[\'(\d{6})\',\'(\D+)\',\d')
        html = ""
        end = False
        while not end:
            s = urllib2.urlopen(self.stock_list_link.substitute(page=cur_page, count=25))
            print ("=========page %-2s=======" % cur_page)
            end = True
            for html in s.readlines():
                # if html is empty, then end is True
                end = False
                html = unicode(html, 'gb2312')
                stock_item_g = stock_item_re.findall(html)
                session = model.getSesstion()
                # save item
                for item in stock_item_g:
                    item_m = Stock(item[0], item[1])
                    item_m.updated_time = datetime.datetime.now()
                    session.add(item_m)
                session.commit()
            cur_page+=1
    
    """
    parse string like 2011年04月22日 to date
    
    """
    def parse_date_string(self, date_string):
        date_g = self.quarter_date_re.search(date_string)
        return datetime.date(int(date_g.group(1)), 
                      int(date_g.group(2)), 
                      int(date_g.group(3)))
    
    def pass_hexun_date_str(self, date_s):
        month, day = 0, 0
        if date_s.find(u'第1季') > 0:
            month, day = 3, 31
        elif date_s.find(u'中期') > 0 or date_s.find(u'第2季') > 0:
            month, day = 6, 30
        elif date_s.find(u'前3季') > 0:
            month, day = 9, 30
        elif date_s.find(u'年度') > 0:
            month, day = 12, 31
        else:
            print('unsupported hexun date string: %s' % date_s)
            raise Exception('error')
        # year is the first 2 chars
        year = date_s[0:2]
        year = int('20' + year)
        return datetime.date(year, month, day)

    def test_pass_hexun_date_str():
        print pass_hexun_date_str(u'11年第1季') 
        print pass_hexun_date_str(u'08年中期')
        print pass_hexun_date_str(u'07年前3季')
        print pass_hexun_date_str(u'06年年度')
    
    """
    Remove % and , from number
    
    """
    def normalize_digital_str(self, s):
        s = re.sub('[,\\%]','',s)
        return s

    def adjust_stock_id(self, stock_id):
        if (stock_id.startswith('6')):
            stock_id = 'sh' + stock_id
        elif (stock_id.startswith('0') or stock_id.startswith('3')):
            stock_id = 'sz' + stock_id
        else:
            raise Exception('Unsupported stock id: %s' % stock_id)
            
        return stock_id
    
    def grab_tradable_holders(self, stock_id):
        _stock_id = self.adjust_stock_id(stock_id)
        s = urllib2.urlopen(self.tradable_top_ten_link.substitute(stockid =_stock_id))
        starting = False
        html = ""
        
        #<title>中信证券(600030)十大流通股东 _ 股票行情 _ 东方财富网</title>
        stock_title_re = re.compile(u'<title>(\D+)\((\d+)\).+')
        stock_date_re = re.compile(u'截止日期：(\S+)\s+公告日期：(\S+)</.*')
        stock_item_re = re.compile(u'<div class="tishi">(\S+)</div></td><td class="(solidborder|bgcolor)">(\d+)</td><td class="(solidborder|bgcolor)">(\S+)%</td><td')
        
        
        for html in s.readlines():
           
            if (html.startswith('<title>')):
                html = unicode(html, 'gb2312')
                result = stock_title_re.search(html)
                stock_name = result.group(1)
                stock_id = result.group(2)
                print(stock_name)
    
            # only grab the tradable stockholders
            if (html.find('<ul id="main0">') > 0):
                starting = True
            if (html.find('<li><div class="tdate">') > 0 and starting):
                
                # info for all years
                all_info = html.split('<div class="tdate">')
                print(len(all_info))
    
                
                for one_quarter in all_info:
                    try:
                        one_quarter = unicode(one_quarter, 'gb2312')
                    except UnicodeDecodeError, e:
                        # gb2312 only covers 6000+ hanzi
                        one_quarter = unicode(one_quarter, 'gb18030')
                    if (one_quarter.startswith(u'截止日期')):
                        p = stock_date_re.search(one_quarter)
                        # due date
                        date_s = p.group(1)
                        date = self.parse_date_string(date_s)
                        
                        release_date_s = p.group(2)
                        release_date = self.parse_date_string(release_date_s)
                        
                        report = model.search_quarter_report(stock_id, date)
                        
                        # if quarter report does not exsit
                        if not report:
                            # create report first
                            report = HolderQuarterReport(stock_id, None, None, 1, date, release_date)
                            model.create(report)
                        
                        report_id = report.id
                        
                        ls = stock_item_re.findall(one_quarter)
                        session = model.getSesstion()
                        for i in ls:
                         
                            holder = TradableStockHolder()
                            holder.report_id = report_id
                            holder.holder_name = i[0]
                            holder.amount = int(i[2])
                            holder.percentage = float(i[4])
                            
                            session.add(holder)
                        session.commit()
    
    """
    update holder amount by quarter
    
    _stockid is as sz000006, sh600003
    
    """
    def update_holder_amount(self, stock_id):
        s = urllib2.urlopen(self.holder_amount_link.substitute(stockid = stock_id))
        html = ""
        

        #u'<td style="background-image:none;">11年第1季</td><td>27,010</td><td class="black_5_de">8,621.41</td><td><font color="red">-13.09%</font></td>'
        holder_count_re =re.compile('<td style="background-image:none;">(\S+)</td>\
<td>([\d,]+)</td><td \S+>([\d,.]+)</td>\
<td><font \S+>([\d.\\%-]+)</font>')
        session = model.getSesstion()
        for html in s.readlines():
            if html.find('<?xml version="1.0" encoding="gb2312"?>') > 0:
                #html = unicode(html, 'gb2312')
                holder_count_g = holder_count_re.findall(html)
                for one_quarter in holder_count_g:
                    date_str = unicode(one_quarter[0], 'gb2312')

                    amount = self.normalize_digital_str(one_quarter[1])
                    amount_per_holder = self.normalize_digital_str(one_quarter[2])
                    amount_change_rate = self.normalize_digital_str(one_quarter[3])
                    date = self.pass_hexun_date_str(date_str)
                    #print('date: %s amount: %d amount_change_rate: %f' %\
                    #      (date, int(amount), float(amount_change_rate)))
                    # get report according to the date
                    report = model.search_quarter_report(stock_id, date, session)
                    if report and not report.trader_amount:
                        report.amount_per_trader = amount_per_holder
                        report.trader_amount = amount
                        report.amount_change_rate = amount_change_rate
                    else:
                        print 'skip'
        session.commit()                
                    
                    
def searchStocks():
    stg = StockTraderGrabber()
    page = 0
    size = 25
    cur_num = 1
    while (True):
        results = model.search_stock_with_pagination(page, size)
        result = results.result
        # we should figure out a way to get the exact row numbers
        #or page==1
        if not result.first():
            print 'finished'
            break
        else:
            print 'current page: %s, total page: %s' %\
                  (results.page, results.total)
            print
            
            for s in result:
                stock_id = str(s.id)
                
                print '%d: %s %s' % (cur_num, s.id, s.name)
                latest_rec = model.search_latest_quarter_holder(stock_id)
                if latest_rec:
                    print '%s already exists, the last record is %s' % (s.name, latest_rec.date)
                else:
                    stg.grab_tradable_holders(stock_id)
                
                cur_num+=1
        page+=1

def update_quarter_holder_amount():
    stg = StockTraderGrabber()
    page = 0
    size = 100
    cur_num = 1
    while (True):
        results = model.search_stock_with_pagination(page, size)
        result = results.result
        # we should figure out a way to get the exact row numbers
        #or page==1
        if not result.first():
            print 'finished'
            break
        else:
            print 'current page: %s, total page: %s' %\
                  (results.page, results.total)
            print
            
            for s in result:
                stock_id = str(s.id)
                
                print '%d: %s %s' % (cur_num, s.id, s.name)
                # get url by stock_id          
                stg.update_holder_amount(stock_id)     
                
                cur_num+=1
        page+=1



    
    

if __name__=='__main__':
    stockid='600640'
    
    stg = StockTraderGrabber()
    #stg.grab_tradable_holders(stockid)
    #stg.grabStockList()
    #searchStocks()
    
    # update holder amount
    update_quarter_holder_amount()
    
    #print model.existsStockHolder('000006')
    stg.update_holder_amount(stockid)
    #test_pass_hexun_date_str()               
                      
                
    